'''
Created on 2019年8月23日
@author: sida
'''
import pandas as pd
import jieba
from gensim import corpora,models,similarities
import compute_data_pre as dtpr
from util import Node
import util

def tfIdfcsvFile2Obj(self):
    csvData = pd.read_csv('resource/data.csv')
    print(csvData.index,csvData.columns)
    return csvData
def tfIdf_get_corpus(self):
    csvData = tfIdfcsvFile2Obj(self)
    self_definition_words_set = set()
    texts = [];trivial = ['上交所','深交所','交易所',"DATAEXA"]
    for _,row in csvData.iterrows():
        labels = eval(row['labels(n)'])
        labels = [label for label in labels if not util.strContainSub(trivial,label)]
#         self_definition_words_set.update(labels)
        s = row.iloc[0]+row.iloc[1]+' '.join(labels)
        texts.append(s)
#     self_definition_words_set.discard("DATAEXA_OBJECT")
    for word in self_definition_words_set:
        jieba.add_word(word )
    corpus_segs = [dtpr.seg_sentence(text)for text in texts]
    dictionary = corpora.Dictionary(corpus_segs)
    corpus = [dictionary.doc2bow(doc) for doc in corpus_segs]
    return (corpus,dictionary,csvData)
def tfidfSimilarity(self):
    corpus,dictionary,_= self.sunNodes[0].getData()
    print(corpus[1],corpus[2],sep='\n')
    tfidf = models.TfidfModel(corpus)
    tfidfS = similarities.SparseMatrixSimilarity(tfidf[corpus], num_features=len(dictionary.keys()) )
    return tfidfS
def inference_tfidfSimilarity(self):
    TopN = 6
    tfidfS = self.sunNodes[0].getData()
    corpus,_,index2Name = self.sunNodes[1].getData()
    tfidfSimiValue = tfidfS[corpus[2]]
    keysSort = sorted(enumerate(tfidfSimiValue), key=lambda item: -item[1])[:TopN]
    print(keysSort)
    [print( index2Name.iloc[key[0]] )for key in keysSort]
    

tfidfSimilarity_path = 'cache/tfidfSimilarity'
node_get_corpus = Node(name='tfIdf_get_corpus',compute=tfIdf_get_corpus,persistence_path='cache/tfIdf_get_corpus')
node_tfidfSimilarity = Node(name='tfidfSimilarity',compute=tfidfSimilarity,sunNodes=[node_get_corpus],persistence_path=tfidfSimilarity_path)
node_inference_tfidfSimilarity = Node(name='inference_tfidfSimilarity',compute=inference_tfidfSimilarity,sunNodes=[node_tfidfSimilarity,node_get_corpus],forceUpdate=True,save=False)

if __name__ == '__main__':
    compulsory_Update_Nodes = [ node_inference_tfidfSimilarity ]
    NoForce_Update_Nodes = []
    wanted_Nodes = NoForce_Update_Nodes + compulsory_Update_Nodes
    for node in compulsory_Update_Nodes:
        node.forceUpdate=True
    [print(type(node.getData())) for node in wanted_Nodes]
    
    
    
